# coding: utf8

import os
import logging
import time
import re
import json
from lxml import etree
import traceback
import requests_html
from multiprocessing.pool import ThreadPool

from copy import deepcopy

"""浙江大学
微电子学院
"""

logging.basicConfig(level=logging.INFO)

session = requests_html.HTMLSession()
pool = ThreadPool(os.cpu_count() * 2)  # 要使用 hanlp, 线程数设为1

sex_ptn = re.compile(r'([男|女])')
degree_ptn = re.compile(r'学位[:：](\S+)')
dep_ptn = re.compile(r'所在单位[:：](\S+)')
subj_ptn = re.compile(r'学科[:：](\S+)')

addr_ptn = re.compile(r'(?:通信地址|通讯地址|办公地址|办公地点|Address)[:：]([ \S]+)')
tel_ptn = re.compile(r'(?:办公电话|电话|联系方式|Tel|tel)[:：]([0-9+\- ,]+)')
email_ptn = re.compile(r'(?:电子邮箱|电子信箱|E-mail|Email|email|e-mail|邮箱地址|邮\s*箱)[:：](\S+@[.\S]+)')
ri_ptn = re.compile(r'研究(?:方向|领域|重点)(?:为|包括)*[:：\n]*([ \n\S]+?[。\n])')
pos_ptn = re.compile(r"(教授|副教授|助理教授|研究员|副研究员|博士导师|硕士导师|博士生研究生|硕士生导师|IEEE Fellow|IEEE\s*会员|博士|讲师|教师|高级工程师|工程师)")
edu_ptn = re.compile(r"(?:考入|攻读|进修|毕业于|学位|博士后|访问|工程师)")
split_ptn = re.compile(r'([;；。\n])')


def teacher_list():
    headers = {"Accept": "application/json",
               "Content-Type": "application/x-www-form-urlencoded; charset=UTF-8",
               "Cookie": "Hm_lvt_fe30bbc1ee45421ec1679d1b8d8f8453=1620753387; Hm_lpvt_fe30bbc1ee45421ec1679d1b8d8f8453=1620753387; JSESSIONID=24A139B49776E5F86E93A380ADD5EC0C",
               "Referer": "http://www.isee.zju.edu.cn/szjs1/list.htm"}

    params = {
        "siteId": 411,
        "columnId": 21034,
        "pageIndex": 1,
        "rows": 10,
        "conditions": [{"orConditions": [{"field": "f2", "value": "%微纳电子学院%", "judge": "like"}]}],
        "orders": [{"field": "letter", "type": "acs"}, {"field": "title", "type": "acs"}],
        "returnInfos": [{"field": "title", "name": "title"}, {"field": "publishTime", "name": "publishTime"}, {"field": "content", "name": "content"}, {"field": "summary", "name": "summary"}, {"field": "subTitle", "name": "subTitle"}, {"field": "f1", "name": "f1"}, {"field": "f2", "name": "f2"}, {"field": "f3", "name": "f3"}, {"field": "f4", "name": "f4"}, {"field": "f5", "name": "f5"}, {"field": "f6", "name": "f6"}, {"field": "mircImgPath", "name": "mircImgPath"}, {"field": "imgPath", "name": "imgPath"}, {"field": "url", "name": "url"}, {"field": "visitCount", "name": "visitCount"}],
        "articleType": 1
    }

    tot_pages = {"value": 0}  # python tricky
    page_idx = 1

    def get_list(_params):
        logging.info("loading page: %s", page_idx)

        _params['pageIndex'] = page_idx

        for key, item in _params.items():
            _params[key] = repr(item)

        try:
            r = session.post("http://www.isee.zju.edu.cn/_wp3services/generalQuery?queryObj=articles", headers=headers, data=_params)
            if r.status_code == 200:
                data = r.json()
                if data.get("status", 0) != 1:
                    logging.warning("error response : %s", r.text)
                    return

                tot_pages['value'] = data.get("pageCount", 0)

                lis = data.get("data", [])
                for li in lis:
                    # logging.info(li)
                    name = li['title']
                    pos = li['f1']
                    dep = li['f2']
                    ri = li['f3']
                    tel = li['f4']
                    email = li['f5']
                    addr = li['f6']
                    avatar = "http://www.isee.zju.edu.cn/" + li['imgPath']
                    homepage = li['subTitle']

                    if homepage and homepage.strip() != "" and homepage.startswith("http"):
                        pool.apply_async(teacher_detail, (name, homepage, pos, dep, ri, tel, email, addr, avatar))
                    pass
                pass
            else:
                logging.warning("wrong status code: %s", r.status_code)
                r.raise_for_status()
        except Exception as e:
            traceback.print_exc()
            raise e

    get_list(deepcopy(params))
    while page_idx < tot_pages['value']:
        page_idx += 1
        get_list(deepcopy(params))


page_uid_ptn = re.compile(r"var pageUid = '(\d+)';")
api_column_ptn = re.compile(r'var apiColumn = "(\S+)";')


def teacher_detail(name, url, *args):
    logging.info("loading %s: %s ...", name, url)

    try:
        r = session.get(url)
        if r.status_code == 200:
            html = r.text

            page_uid, api_column = [ptn.search(html) for ptn in (page_uid_ptn, api_column_ptn)]
            page_uid, api_column = [(match.group(1) if match else None) for match in (page_uid, api_column)]

            if not page_uid or not api_column:
                logging.warning("%s api url error: %s, %s", page_uid, api_column)
                return

            api_url = r.html.base_url + api_column + "&column_id=0" + "&pageUid=" + page_uid + "&type=1"
            logging.info("name: %s, api_url: %s", name, api_url)

            r1 = session.get(api_url)  # headers={"Accept": "application/json"}
            if r1.status_code != 200:
                logging.warning("%s api req error: %s", name, r1.status_code)
                return

            data = json.loads(r1.html.full_text)

            if data['code'] != 200:
                logging.warning("%s api response error: %s", name, data)
                return

            content = data['data']['content']
            content = etree.HTML(content)

            intro = "".join(content.xpath("//*/text()"))

            logging.info("name: %s, others: %s", name, args)
            pass
        else:
            logging.warning("wrong status code: %s", r.status_code)
            r.raise_for_status()

    except Exception as e:
        traceback.print_exc()
        raise e


if __name__ == '__main__':
    teacher_list()

    pool.close()
    pool.join()

    session.close()

    pass

